package org.solrcn.segment;

import java.io.IOException;
import java.io.Reader;
import java.io.File;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.StringReader;
import java.lang.StringBuilder;
import java.io.BufferedWriter;
import java.io.FileWriter;

public class SegChinese{
	private final static String PUNCTION = "。，！？；,!?;";
	public static final String SPACES = " 　\t\r\n";

	public static int getCharType(char ch) {
		// Most (but not all!) of these are Han Ideographic Characters
		if (ch >= 0x4E00 && ch <= 0x9FA5)
			return CharType.HANZI;
		if ((ch >= 0x0041 && ch <= 0x005A) || (ch >= 0x0061 && ch <= 0x007A))
			return CharType.LETTER;
		if (ch >= 0x0030 && ch <= 0x0039)
			return CharType.DIGIT;
		if (ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '　')
			return CharType.SPACE_LIKE;
		// Punctuation Marks
		if ((ch >= 0x0021 && ch <= 0x00BB) || (ch >= 0x2010 && ch <= 0x2642)
				|| (ch >= 0x3001 && ch <= 0x301E))
			return CharType.DELIMITER;

		// Full-Width range
		if ((ch >= 0xFF21 && ch <= 0xFF3A) || (ch >= 0xFF41 && ch <= 0xFF5A))
			return CharType.FULLWIDTH_LETTER;
		if (ch >= 0xFF10 && ch <= 0xFF19)
			return CharType.FULLWIDTH_DIGIT;
		if (ch >= 0xFE30 && ch <= 0xFF63)
			return CharType.DELIMITER;
		return CharType.OTHER;

	}

	public static void outputbuffer( StringBuilder buffer, String countfilename ) throws IOException
	{
		System.out.println( buffer );

		File f = new File( countfilename );
		if( !f.exists() )
		{
			f.createNewFile();
		}

		BufferedWriter output = new BufferedWriter(new FileWriter(f, true));

		int nGram = 5;
		int bufferLen = buffer.length();
		String strContent = "";
		for( int i = 0; i < bufferLen; i++ )
		{
			for( int j = 0; j < nGram; j++ )
			{
				if( ( i + j ) < bufferLen )
				{
					String subNgram = buffer.substring( i, i + j + 1 );
					System.out.println( subNgram );
					strContent += subNgram + ",1\n";
				}
			}
		}

		output.write( strContent );
		output.flush();
		output.close();
	}

	public static void processafile( String filename, String countfilename )
	{
		//first:read content from file
		File f = new File( filename );
		BufferedReader reader = null;

		try{
			reader = new BufferedReader(new FileReader(f));
			String tempString = null;
			StringBuilder buffer = new StringBuilder();
			buffer.setLength( 0 );
			while ((tempString = reader.readLine()) != null)
			{
				Reader r = new StringReader( tempString );
				int ci;
				char ch, pch;
				ci = r.read();
				ch = (char)ci;
				while( true ) {
					if( ci == -1 ){
						break;
					}
					else if( PUNCTION.indexOf( ch ) != -1 || SPACES.indexOf(ch) != -1 ){
						if( buffer.length() != 0 )
						{
							outputbuffer( buffer, countfilename );
						}
						buffer.setLength( 0 );
					}
					else if( getCharType(ch) != CharType.HANZI ){
						if( buffer.length() != 0 )
						{
							outputbuffer( buffer, countfilename );
						}
						buffer.setLength( 0 );
					}
					else{
						buffer.append(ch);
					}

					ci = r.read();
					ch = (char)ci;
				}
				outputbuffer( buffer, countfilename );
			}

			reader.close();
			reader = null;
		}catch(IOException e){
			e.printStackTrace();
			return;
		}finally {
			if (reader != null) {
				try {
					reader.close();
				} catch (IOException e1) {
				}
			}
		}
	}

	public static void main( String[] args ) throws IOException
	{
		String filename = "/tmp/solr2file/366";
		File dir = new File( "G:/news" );
		File file[] = dir.listFiles();
		//processafile( "/tmp/solr2file/366", "/tmp/count/3721.txt" );
		for (int i = 0; i < file.length; i++)
		{
			if (!file[i].isDirectory())
			{
				System.out.println( file[i].getPath() );
				System.out.println( file[i].getName() );
				processafile( file[i].getPath(), "G:/tmp/" + file[i].getName() + ".txt" );
			}
		}
	}

}
